Guided Project: Guided Project: Predicting House Sale Prices
Posted on Wed 08 July 2015 in Projects
Introduction¶
In [1]:
import pandas as pd
pd.options.display.max_columns = 999
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn import linear_model
from sklearn.model_selection import KFold
In [2]:
df = pd.read_csv("AmesHousing.tsv", delimiter="\t")
In [3]:
def transform_features(df):
return df
def select_features(df):
return df[["Gr Liv Area", "SalePrice"]]
def train_and_test(df):
train = df[:1460]
test = df[1460:]
## You can use `pd.DataFrame.select_dtypes()` to specify column types
## and return only those columns as a data frame.
numeric_train = train.select_dtypes(include=['integer', 'float'])
numeric_test = test.select_dtypes(include=['integer', 'float'])
## You can use `pd.Series.drop()` to drop a value.
features = numeric_train.columns.drop("SalePrice")
lr = linear_model.LinearRegression()
lr.fit(train[features], train["SalePrice"])
predictions = lr.predict(test[features])
mse = mean_squared_error(test["SalePrice"], predictions)
rmse = np.sqrt(mse)
return rmse
transform_df = transform_features(df)
filtered_df = select_features(transform_df)
rmse = train_and_test(filtered_df)
rmse
Out[3]:
Feature Engineering¶
- Handle missing values:
- All columns:
- Drop any with 5% or more missing values for now.
- Text columns:
- Drop any with 1 or more missing values for now.
- Numerical columns:
- For columns with missing values, fill in with the most common value in that column
- All columns:
1: All columns: Drop any with 5% or more missing values for now.
In [4]:
## Series object: column name -> number of missing values
num_missing = df.isnull().sum()
In [5]:
# Filter Series to columns containing >5% missing values
drop_missing_cols = num_missing[(num_missing > len(df)/20)].sort_values()
# Drop those columns from the data frame. Note the use of the .index accessor
df = df.drop(drop_missing_cols.index, axis=1)
2: Text columns: Drop any with 1 or more missing values for now.
In [6]:
## Series object: column name -> number of missing values
text_mv_counts = df.select_dtypes(include=['object']).isnull().sum().sort_values(ascending=False)
## Filter Series to columns containing *any* missing values
drop_missing_cols_2 = text_mv_counts[text_mv_counts > 0]
df = df.drop(drop_missing_cols_2.index, axis=1)
3: Numerical columns: For columns with missing values, fill in with the most common value in that column
In [7]:
## Compute column-wise missing value counts
num_missing = df.select_dtypes(include=['int', 'float']).isnull().sum()
fixable_numeric_cols = num_missing[(num_missing < len(df)/20) & (num_missing > 0)].sort_values()
fixable_numeric_cols
Out[7]:
In [8]:
## Compute the most common value for each column in `fixable_nmeric_missing_cols`.
replacement_values_dict = df[fixable_numeric_cols.index].mode().to_dict(orient='records')[0]
replacement_values_dict
Out[8]:
In [9]:
## Use `pd.DataFrame.fillna()` to replace missing values.
df = df.fillna(replacement_values_dict)
In [10]:
## Verify that every column has 0 missing values
df.isnull().sum().value_counts()
Out[10]:
What new features can we create, that better capture the information in some of the features?
In [11]:
years_sold = df['Yr Sold'] - df['Year Built']
years_sold[years_sold < 0]
Out[11]:
In [12]:
years_since_remod = df['Yr Sold'] - df['Year Remod/Add']
years_since_remod[years_since_remod < 0]
Out[12]:
In [13]:
## Create new columns
df['Years Before Sale'] = years_sold
df['Years Since Remod'] = years_since_remod
## Drop rows with negative values for both of these new features
df = df.drop([1702, 2180, 2181], axis=0)
## No longer need original year columns
df = df.drop(["Year Built", "Year Remod/Add"], axis = 1)
Drop columns that:
- that aren't useful for ML
- leak data about the final sale, read more about columns here
In [14]:
## Drop columns that aren't useful for ML
df = df.drop(["PID", "Order"], axis=1)
## Drop columns that leak info about the final sale
df = df.drop(["Mo Sold", "Sale Condition", "Sale Type", "Yr Sold"], axis=1)
Let's update transform_features()
In [15]:
def transform_features(df):
num_missing = df.isnull().sum()
drop_missing_cols = num_missing[(num_missing > len(df)/20)].sort_values()
df = df.drop(drop_missing_cols.index, axis=1)
text_mv_counts = df.select_dtypes(include=['object']).isnull().sum().sort_values(ascending=False)
drop_missing_cols_2 = text_mv_counts[text_mv_counts > 0]
df = df.drop(drop_missing_cols_2.index, axis=1)
num_missing = df.select_dtypes(include=['int', 'float']).isnull().sum()
fixable_numeric_cols = num_missing[(num_missing < len(df)/20) & (num_missing > 0)].sort_values()
replacement_values_dict = df[fixable_numeric_cols.index].mode().to_dict(orient='records')[0]
df = df.fillna(replacement_values_dict)
years_sold = df['Yr Sold'] - df['Year Built']
years_since_remod = df['Yr Sold'] - df['Year Remod/Add']
df['Years Before Sale'] = years_sold
df['Years Since Remod'] = years_since_remod
df = df.drop([1702, 2180, 2181], axis=0)
df = df.drop(["PID", "Order", "Mo Sold", "Sale Condition", "Sale Type", "Year Built", "Year Remod/Add"], axis=1)
return df
def select_features(df):
return df[["Gr Liv Area", "SalePrice"]]
def train_and_test(df):
train = df[:1460]
test = df[1460:]
## You can use `pd.DataFrame.select_dtypes()` to specify column types
## and return only those columns as a data frame.
numeric_train = train.select_dtypes(include=['integer', 'float'])
numeric_test = test.select_dtypes(include=['integer', 'float'])
## You can use `pd.Series.drop()` to drop a value.
features = numeric_train.columns.drop("SalePrice")
lr = linear_model.LinearRegression()
lr.fit(train[features], train["SalePrice"])
predictions = lr.predict(test[features])
mse = mean_squared_error(test["SalePrice"], predictions)
rmse = np.sqrt(mse)
return rmse
df = pd.read_csv("AmesHousing.tsv", delimiter="\t")
transform_df = transform_features(df)
filtered_df = select_features(transform_df)
rmse = train_and_test(filtered_df)
rmse
Out[15]:
Feature Selection¶
In [16]:
numerical_df = transform_df.select_dtypes(include=['int', 'float'])
numerical_df.head(5)
Out[16]:
In [17]:
abs_corr_coeffs = numerical_df.corr()['SalePrice'].abs().sort_values()
abs_corr_coeffs
Out[17]:
In [18]:
## Let's only keep columns with a correlation coefficient of larger than 0.4 (arbitrary, worth experimenting later!)
abs_corr_coeffs[abs_corr_coeffs > 0.4]
Out[18]:
In [19]:
## Drop columns with less than 0.4 correlation with SalePrice
transform_df = transform_df.drop(abs_corr_coeffs[abs_corr_coeffs < 0.4].index, axis=1)
Which categorical columns should we keep?
In [20]:
## Create a list of column names from documentation that are *meant* to be categorical
nominal_features = ["PID", "MS SubClass", "MS Zoning", "Street", "Alley", "Land Contour", "Lot Config", "Neighborhood",
"Condition 1", "Condition 2", "Bldg Type", "House Style", "Roof Style", "Roof Matl", "Exterior 1st",
"Exterior 2nd", "Mas Vnr Type", "Foundation", "Heating", "Central Air", "Garage Type",
"Misc Feature", "Sale Type", "Sale Condition"]
- Which columns are currently numerical but need to be encoded as categorical instead (because the numbers don't have any semantic meaning)?
- If a categorical column has hundreds of unique values (or categories), should we keep it? When we dummy code this column, hundreds of columns will need to be added back to the data frame.
In [21]:
## Which categorical columns have we still carried with us? We'll test tehse
transform_cat_cols = []
for col in nominal_features:
if col in transform_df.columns:
transform_cat_cols.append(col)
## How many unique values in each categorical column?
uniqueness_counts = transform_df[transform_cat_cols].apply(lambda col: len(col.value_counts())).sort_values()
## Aribtrary cutoff of 10 unique values (worth experimenting)
drop_nonuniq_cols = uniqueness_counts[uniqueness_counts > 10].index
transform_df = transform_df.drop(drop_nonuniq_cols, axis=1)
In [22]:
## Select just the remaining text columns and convert to categorical
text_cols = transform_df.select_dtypes(include=['object'])
for col in text_cols:
transform_df[col] = transform_df[col].astype('category')
## Create dummy columns and add back to the dataframe!
transform_df = pd.concat([
transform_df,
pd.get_dummies(transform_df.select_dtypes(include=['category']))
], axis=1).drop(text_cols,axis=1)
Update select_features()
In [23]:
def transform_features(df):
num_missing = df.isnull().sum()
drop_missing_cols = num_missing[(num_missing > len(df)/20)].sort_values()
df = df.drop(drop_missing_cols.index, axis=1)
text_mv_counts = df.select_dtypes(include=['object']).isnull().sum().sort_values(ascending=False)
drop_missing_cols_2 = text_mv_counts[text_mv_counts > 0]
df = df.drop(drop_missing_cols_2.index, axis=1)
num_missing = df.select_dtypes(include=['int', 'float']).isnull().sum()
fixable_numeric_cols = num_missing[(num_missing < len(df)/20) & (num_missing > 0)].sort_values()
replacement_values_dict = df[fixable_numeric_cols.index].mode().to_dict(orient='records')[0]
df = df.fillna(replacement_values_dict)
years_sold = df['Yr Sold'] - df['Year Built']
years_since_remod = df['Yr Sold'] - df['Year Remod/Add']
df['Years Before Sale'] = years_sold
df['Years Since Remod'] = years_since_remod
df = df.drop([1702, 2180, 2181], axis=0)
df = df.drop(["PID", "Order", "Mo Sold", "Sale Condition", "Sale Type", "Year Built", "Year Remod/Add"], axis=1)
return df
def select_features(df, coeff_threshold=0.4, uniq_threshold=10):
numerical_df = df.select_dtypes(include=['int', 'float'])
abs_corr_coeffs = numerical_df.corr()['SalePrice'].abs().sort_values()
df = df.drop(abs_corr_coeffs[abs_corr_coeffs < coeff_threshold].index, axis=1)
nominal_features = ["PID", "MS SubClass", "MS Zoning", "Street", "Alley", "Land Contour", "Lot Config", "Neighborhood",
"Condition 1", "Condition 2", "Bldg Type", "House Style", "Roof Style", "Roof Matl", "Exterior 1st",
"Exterior 2nd", "Mas Vnr Type", "Foundation", "Heating", "Central Air", "Garage Type",
"Misc Feature", "Sale Type", "Sale Condition"]
transform_cat_cols = []
for col in nominal_features:
if col in df.columns:
transform_cat_cols.append(col)
uniqueness_counts = df[transform_cat_cols].apply(lambda col: len(col.value_counts())).sort_values()
drop_nonuniq_cols = uniqueness_counts[uniqueness_counts > 10].index
df = df.drop(drop_nonuniq_cols, axis=1)
text_cols = df.select_dtypes(include=['object'])
for col in text_cols:
df[col] = df[col].astype('category')
df = pd.concat([df, pd.get_dummies(df.select_dtypes(include=['category']))], axis=1).drop(text_cols,axis=1)
return df
def train_and_test(df, k=0):
numeric_df = df.select_dtypes(include=['integer', 'float'])
features = numeric_df.columns.drop("SalePrice")
lr = linear_model.LinearRegression()
if k == 0:
train = df[:1460]
test = df[1460:]
lr.fit(train[features], train["SalePrice"])
predictions = lr.predict(test[features])
mse = mean_squared_error(test["SalePrice"], predictions)
rmse = np.sqrt(mse)
return rmse
if k == 1:
# Randomize *all* rows (frac=1) from `df` and return
shuffled_df = df.sample(frac=1, )
train = df[:1460]
test = df[1460:]
lr.fit(train[features], train["SalePrice"])
predictions_one = lr.predict(test[features])
mse_one = mean_squared_error(test["SalePrice"], predictions_one)
rmse_one = np.sqrt(mse_one)
lr.fit(test[features], test["SalePrice"])
predictions_two = lr.predict(train[features])
mse_two = mean_squared_error(train["SalePrice"], predictions_two)
rmse_two = np.sqrt(mse_two)
avg_rmse = np.mean([rmse_one, rmse_two])
print(rmse_one)
print(rmse_two)
return avg_rmse
else:
kf = KFold(n_splits=k, shuffle=True)
rmse_values = []
for train_index, test_index, in kf.split(df):
train = df.iloc[train_index]
test = df.iloc[test_index]
lr.fit(train[features], train["SalePrice"])
predictions = lr.predict(test[features])
mse = mean_squared_error(test["SalePrice"], predictions)
rmse = np.sqrt(mse)
rmse_values.append(rmse)
print(rmse_values)
avg_rmse = np.mean(rmse_values)
return avg_rmse
df = pd.read_csv("AmesHousing.tsv", delimiter="\t")
transform_df = transform_features(df)
filtered_df = select_features(transform_df)
rmse = train_and_test(filtered_df, k=4)
rmse
Out[23]: